
package com.csdn;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.ConnectException;
import java.util.HashMap;
import java.util.Map;
 








import org.apache.log4j.LogManager;
import org.jsoup.*;
import org.jsoup.nodes.*;
import org.jsoup.select.*;
import org.jsoup.select.Evaluator.IsEmpty;
 

 
public class Main {

	private static org.apache.log4j.Logger log = LogManager.getLogger(Main.class);


	// private static final String URL = "http://blog.csdn.net/sunhuaqiang1";
	private static final String expertsURl = "http://blog.csdn.net/peoplelist.html";
	private static final String expertURL = "/peoplelist.html?&page=";
	private static final int settime = 10000;
 

	public static void main(String[] args) throws IOException {


        System.setProperty("http.maxRedirects", "50");
		System.getProperties().setProperty("proxySet", "true");
	//	System.getProperties().setProperty("http.proxyHost", "180.97.250.86");
		System.getProperties().setProperty("http.proxyHost", "183.222.102.101");
		System.getProperties().setProperty("http.proxyPort", "80");

		Connection getAllExperts = Jsoup
				.connect(expertsURl)
				.userAgent(
						"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:50.0) Gecko/20100101 Firefox/50.0")
				.timeout(settime).method(Connection.Method.GET);

		Document ex = getAllExperts.get();
		Element exbody = ex.body();

		String total = exbody.getElementsByClass("page_nav")
				.select("span:eq(0)").text();
		String regex = ".+共(\\d+)页";
		total = total.replaceAll(regex, "$1");
		int totalPage = Integer.parseInt(total);

		getExperts(totalPage);
     // getblog("http://blog.csdn.net/robertcpp");
		// System.out.println(getTopic("http://blog.csdn.net/u014532717/article/details/59150512"));
	}

	public static void getExperts(int totalPage) throws IOException {
		for (int i = 0; i <= totalPage; i++) {
			Connection getAllExpert = Jsoup
					.connect("http://blog.csdn.net" + expertURL + i)
					.userAgent(UserAgentMiddle.Agent()).timeout(3000)
					.method(Connection.Method.GET);
			Document exe = getAllExpert.get();
			Element ebody = exe.body();
			Elements es = ebody.getElementsByClass("experts_list");
			int count = 0;
			for (Element element : es) {
				Element linkNode = (element.select("dt a")).get(0);
				String url = linkNode.attr("href");
				count ++;
				try {
					if(count ==4){
						System.setProperty("http.maxRedirects", "50");
						System.getProperties().setProperty("proxySet", "true");
						System.getProperties().setProperty("http.proxyHost", "183.222.102.101");
						System.getProperties().setProperty("http.proxyPort", "80");		
						
					}
					if(count ==8){
						System.setProperty("http.maxRedirects", "50");
						System.getProperties().setProperty("proxySet", "true");
						System.getProperties().setProperty("http.proxyHost", "106.119.0.244");
						System.getProperties().setProperty("http.proxyPort", "8080");		
						
					}
					if(count ==14){
						System.setProperty("http.maxRedirects", "50");
						System.getProperties().setProperty("proxySet", "true");
						System.getProperties().setProperty("http.proxyHost", "183.222.102.108");
						System.getProperties().setProperty("http.proxyPort", "8080");		
						count = 0;
					}
					
					getblog(linkNode.attr("href"));
				} catch (RuntimeException e) {
					log.error("没数据："+url);
					continue;
					
				} catch (Exception e) {
					e.printStackTrace();
					 log.error("没爬取："+url +" :" + e.getMessage() +"人页数" +i);
					 count ++;
					 continue;
				}

			}

		}

		
	}

	// 获取他的博客列表
	public static void getblog(String url) throws IOException {

		Connection conn = Jsoup.connect(url).userAgent(UserAgentMiddle.Agent())
				.timeout(settime).method(Connection.Method.GET);
		Document doc = conn.get();
		Element body = doc.body();

		
		Element count = body.getElementById("papelist");
		int totalPage  = 2;
		if(count != null && count.hasText()){
			// 获取总页数
			String totalPageStr = body.getElementById("papelist")
					.select("span:eq(0)").text();
			String regex = ".+共(\\d+)页";
			totalPageStr = totalPageStr.replaceAll(regex, "$1");
			totalPage = Integer.parseInt(totalPageStr);
		}
		
		
		int pageNow = 1;
	
		for (pageNow = 1; pageNow <= totalPage; pageNow++) {
			log.info("博客人:"+url +":第"+pageNow);
			getArtitcleByPage(url, pageNow);
		}
	}

	// 解析他所有博客列表
	public static void getArtitcleByPage(String url, int pageNow)
			throws IOException {
		Connection conn = Jsoup.connect(url + "/article/list/" + pageNow)
				.userAgent(UserAgentMiddle.Agent()).timeout(settime)
				.method(Connection.Method.GET);
		Document doc = conn.get();
		Element body = doc.body();

		Element articleListDiv = body.getElementById("skin_center");
	     boolean flag = false;
		if(articleListDiv == null || !articleListDiv.hasText()){
			articleListDiv  = body.getElementById("article_list");
			flag =true;
			
		}
		Elements articleList = null;
		if(flag){
			articleList =  articleListDiv.getElementsByClass("list_item");
		}else{
			articleList  = articleListDiv.getElementsByClass("list_c_t");
		}
		if(!articleList.hasText()){
			throw  new RuntimeException("没数据");
		}
		Element linkNode = null;
		Elements ls =  null;
		Document bodys = null;
		int count =0;
		for (Element article : articleList) {
			try {
				count ++;
				ls = article.select("h3 a");
				if(ls.isEmpty()){
					ls = article.select("div h1 a");
				}
				if(count ==4){
					System.setProperty("http.maxRedirects", "50");
					System.getProperties().setProperty("proxySet", "true");
					System.getProperties().setProperty("http.proxyHost", "183.222.102.108");
					System.getProperties().setProperty("http.proxyPort", "8080");	
				}
				if(count ==8){
					System.setProperty("http.maxRedirects", "50");
					System.getProperties().setProperty("proxySet", "true");
					System.getProperties().setProperty("http.proxyHost", "183.222.102.107");
					System.getProperties().setProperty("http.proxyPort", "80");	
				}
				if(count ==12){
					System.setProperty("http.maxRedirects", "50");
					System.getProperties().setProperty("proxySet", "true");
					System.getProperties().setProperty("http.proxyHost", "183.222.102.105");
					System.getProperties().setProperty("http.proxyPort", "80");	
				}
				if(count ==16){
					System.setProperty("http.maxRedirects", "50");
					System.getProperties().setProperty("proxySet", "true");
					System.getProperties().setProperty("http.proxyHost", "183.222.102.94");
					System.getProperties().setProperty("http.proxyPort", "8080");	
					count= 0;
				}
				 
				linkNode = ls.get(0);
				 bodys = getdetails(linkNode.attr("href"));
				connectionUrl(linkNode.text(), getTopic(bodys),
						url.substring(url.lastIndexOf("/") + 1, url.length()), 2,
						5, null, "fxbbs",getfourmName(bodys));
			} catch (ConnectException e1) {// 超时重试
				e1.printStackTrace();
				log.info(tagRest+":connectionUrl:"+  ls.get(0));
				tagRest ++;
				System.setProperty("http.maxRedirects", "50");
				System.getProperties().setProperty("proxySet", "true");
				System.getProperties().setProperty("http.proxyHost", "183.222.102.106");
				System.getProperties().setProperty("http.proxyPort", "8080");	 
				connectionUrl(linkNode.text(), getTopic(bodys),
						url.substring(url.lastIndexOf("/") + 1, url.length()), 2,
						5, null, "fxbbs",getfourmName(bodys));
				if(tagRest == 2){
					System.setProperty("http.maxRedirects", "50");
					System.getProperties().setProperty("proxySet", "true");
					System.getProperties().setProperty("http.proxyHost", "106.119.0.244");
					System.getProperties().setProperty("http.proxyPort", "8080");	 
					connectionUrl(linkNode.text(), getTopic(bodys),
							url.substring(url.lastIndexOf("/") + 1, url.length()), 2,
							5, null, "fxbbs",getfourmName(bodys));
					tagRest = 0;
				}
			
			}catch (Exception e) {
				e.printStackTrace();
				count ++;
				log.error("重试"+count+" url:"+url);
				log.error(e.getMessage());
				continue;
			}
			

		}

	}
	static int tagRest = 0;
	private static String getfourmName(Document bodys) {
		Elements es = bodys.getElementsByClass("category_r");
		if (es.hasText()) {
			for (Element element : es) {
				Element linkNode = null;
				Elements ls = null;
				try {
					ls = element.select("label span");
					if(ls.isEmpty()){
						ls = element.select("label em");
					}
					linkNode = ls.get(0);
				} catch (Exception e) {
					e.printStackTrace();
					return null;
				}
				String fournName = linkNode.text();
				if (fournName.contains("（")) {
					fournName = fournName.substring(0, fournName.indexOf("（"));
				} else if (fournName.contains("(")) {
					fournName = fournName.substring(0, fournName.indexOf("("));
				}
				return fournName;
			}

		} else {
			es = bodys.getElementsByClass("detail_p");
			for (Element element : es) {

				Element linkNode = null;
				try {
					linkNode = (element.select("label em")).get(0);
				} catch (Exception e) {
					return null;
				}

				String fournName = linkNode.text();
				if (fournName.contains("（")) {
					fournName = fournName.substring(0, fournName.indexOf("（"));
				} else if (fournName.contains("(")) {
					fournName = fournName.substring(0, fournName.indexOf("("));
				}
				return fournName;
			}
		}

		return null;
	}

	// 获取帖子内容
	public static String getTopic(Document bodys) throws IOException {
		Elements es = bodys.getElementsByClass("article_content");
		if(!es.hasText()){
			  es = bodys.getElementsByClass("skin_detail");
		}
		return es.toString();
	}
	private static int count = 0;
	public static Document getdetails(String url) throws IOException {
		Document  body = Jsoup.connect("http://blog.csdn.net" + url)
				.userAgent(UserAgentMiddle.Agent()).timeout(settime)
				.method(Connection.Method.GET).get() ;
		return body;
	}
	
	
	

	public static void connectionUrl(String title, String content,
			String userName, Integer forumId, Integer postTypeId,
			Integer category, String ps,String fourmName) throws UnsupportedEncodingException {

		Map<String, String> params = new HashMap<String, String>();
		params.put("title", title);
		params.put("content", content);
		params.put("userName", userName);

		params.put("forumId", forumId + "");
		params.put("postTypeId", postTypeId + "");
		params.put("ps", ps);
		params.put("froumName", fourmName == null ?"1":fourmName);
		String result = HttpUtils.sendPostMessage(params, "UTF-8");
		log.debug("-result->>" + result);
 
	}

}